# 云筑集采的供应商信息爬虫
import random
import time
import pymysql
import redis
import requests
from pkg.msg_queue import connect_message_queue
from pkg.user_agent_kuaidaili_proxy import get_random_ua
from pkg.md5 import MD5Utils
import datetime

redis_test_url = "redis://192.168.1.9:6379/5"
mysql45_config = {
            'host': "192.168.1.45",
            'user': "xzl",
            'password': "Xzl4kyk!",
            'database': "qcc_company_graph",
            'port': 3306,
            'charset': 'utf8mb4'
        }
URL = 'https://xy.yzw.cn/api/xy/mer/portal/supplier/search'
proxy = {'http': 'http://t10635913038651:09ht71vf@tps152.kdlapi.com:15818/', 'https': 'http://t10635913038651:09ht71vf@tps152.kdlapi.com:15818/'}
REDIS_PASSWORD = None


class YzwSpiderProduce(object):
    def __init__(self):
        self.company_source = 'www.yzw.cn'
        self.redis_url = redis_test_url
        self.init_url = URL
        self.mysql_conn = pymysql.connect(**mysql45_config)
        self.mysql_cursor = self.mysql_conn.cursor()
        self.redis_conn = redis.Redis(host='localhost', port=6379, db=5, password=REDIS_PASSWORD)

    def distinct_id_redis(self, company_id):
        if self.redis_conn.sismember('yzw_company_set', company_id):
            print('company_id已存在跳过')
            return True
        else:
            self.redis_conn.sadd('yzw_company_set', company_id)
            return False

    def distinct_retry_data_redis(self, url):
        if self.redis_conn.sismember('retry_yzw_company_set', url):
            print('retry_url已存在跳过')
            return True
        else:
            self.redis_conn.sadd('retry_yzw_company_set', url)
            return False

    def save_mysql(self,company_name, company_id, address, register_date, register_fund):
        # 查询名称是否已存在
        select_sql = "SELECT * FROM company_info WHERE company_id='%s'" % company_id
        self.mysql_cursor.execute(select_sql)
        result = self.mysql_cursor.fetchone()

        if result:
            # 找到重复数据,跳过插入
            print("%s公司已存在" % company_name)
            return False

        insert_sql = "INSERT INTO company_info (source, company_name, company_id, address, register_date, register_fund, create_on) " \
                     "VALUES (%s, %s, %s, %s, %s, %s, NOW())"
        self.mysql_cursor.execute(insert_sql, (self.company_source,company_name, company_id, address, register_date, register_fund))
        self.mysql_conn.commit()
        return True


    def request_get_method(self, data):
        # 获取列表页图片数据
        new_user_agent = get_random_ua()
        headers = {
            'Accept': 'application/json, text/plain, */*',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Boxid': 'BIAU+R2mHoNSyqzMuuaJ4qzbyfXSzRFQvBa+XbMQqYcLgvpHishgMypZ/7wnoAL33xz0GL86MOkSQKRxZjyfGUA==',
            'Content-Length': '84',
            'Content-Type': 'application/json',
            'Cookie': 'Hm_lvt_c1832dca4922753f642109f295f07eba=1705455589; yzw-auac-token=7T37OMnWmNbc9GWB9yrP3UC%2FNMdk8NTOHTsBbiBL8RTaALJmTE5mSAGmwG3FySNBxwA6ael7TbSFR%2FTDaE8f4A%3D%3D; LoginRequestKey=AE16E63E1E6811D5FB9C6E3EDB975FE5C465C88EB9378271E0793311458E409A286D5E7C7FDDA90BA3AFDBE59925BB91FE4B85BEE9EDB4EC096337ED9F6E1D83B591DC2277385A0CC67F8BFD6105046C15F3CF405F18CC519D412A31A8FB95E38827A62F6AB7B779FC757578E95D289A1691022BBB1BF307C3137727A8C27C32; web.auth.yzw=5D0526AB1514EF56226ADF6C73DFFE651EFB110AF325EE3ABD3B535C69906CE725085D7208A8F76FFACD9BD3D3079E9A8310CDCCB1C19CF1031D5B8CD95056A8939AFD057416591CCA52DECA175E67F59A587EBC8857B332955E3A17F5D496D1B20CB13678F8A552FF601B2A96392FCC56943902B731A532833326619FC5754784907DD5381CE5A81263E63D16E42BF52BA2829D2AC317E516ABA6F22683A30010CF0FF8FF0F83367EA114F58A101E2AEBA051528D248B8E91F6AF7FD4683CF0C1FF4807A3F3AA35D38701546745CB996321D086AA0CE1B5283803249A396628B42892A223A7561A0141341BC6CD042D63B2079811BF2DC5519FB113498E8CF889AA2B300A92324CA8024BAAD18B73D0866C6AF46E9A3341922631876BE524BF23003EEC50C1E399788BB430D15842885C4813FAF461806D931798471E7CD70FA7AE6114A3290D6EFB7DFF9EF4259F76484B36D0E271F146B64065DECCBC4607AF5D29AA6F3B21ADA171E59EC1E8DC9BCD968C65BAFCE0C427506308DA27C8FA887B1FEF7B371DBEADFB1D751BE19897A00F94C43849264974B494A0637369C09E2E0F9DC50C0238969F02FD9A725931A225340019C37E87A51F20060319A0224C521B8739091BBC8B9F89F34948CB6504C15C128F32B8A363C086578817A6B5B6F7F3FE52E600B334A73584C21134F1B3CADE83A3140C41C0D5FF4BAF19B53E0708ED2EBD198FD61124845F7A55BAA8592F6F88942CE90E462293401F24CBFA1BAFAB878F5FB53C4203DC569FD7FBED17A36715F4298B3496BE2BB5EBFC66721DC82FBD592965F8E00AEF9C7997D549C0D8C0AF98129FA43A10A0C47D50A12F83492FB7C4CA4A3833691EAA171B907F55FFEA1528C6F4DC30E95399EE2634808C583C8E56AD17E33B0004DE9969EB5D6C41696FF69DD0521B4466CB85F23ECE52BB26599C95E54DEC8C48305BB8654EF0EA79C1A31D22F101C4CA06F7E7CDBFFE8159C2BD979AE3B8A4FA76ADDB58781AF98B75363AE59F3703E595658D2E4E3AEAF5FFA3CA6D2F3F056D9B24CC; Hm_lpvt_c1832dca4922753f642109f295f07eba=1705465728; yzw-oauth2-access-token=8ab468c2-8946-4885-8405-db697bd5b4e0; HWWAFSESID=730c173f1eb9677534; HWWAFSESTIME=1705465778617; smidV2=20240117122939e8148f1bc728a999609d7f96bb8f6cf500cfdbbbbf7cf8780; Hm_lvt_3b316cea958b1e6fb00e41da555d5324=1705465781; LANGUAGE_CODE=546F9E8B927D7D8F932CE4A02BC4C267332F8BDEE9E6B4E6827BF8; jf_user_selectedOrg=BxzMoBlmEKozBEUExLBCvtBRFYoZlPCv7QjMbH7dvJr9kVugo8SIfPB1Bn3FHKP4Uo09Ew2zhNPBZRXY/5+4qEauPDBPc1YsQl1Rv/1XCSfW3Mbn5UsKd9f2F5557wlIMo0NuiycsCIulJMRRpACfVfsAh1eDZTmWlzg5xoPXMJAc1G4+yhtDFKWz+JRSHdwn5HB048lxnTIpbnQz7M9iwLxN7KBxc+t6bEIUvC6JvfPMZvET8KSn8wWJPoNALqQ0PZ1CHaaJI7V/AQwJ2Du3FnGTbKCYo0M2C6Y6STGY1wwPc1p0Lz70WcShupvEhhZ; weakpwd.yzw=6586433E2D4284A2527A069BFD39726FCB5376D7EE66D73ABD; .thumbcache_44d09cf98d9aa5ab213486c7b33a5142=cNBRyRoKA3MRA/4F82aSKh+FTVgM+U4juiPdAyfkqNDkFoTPk5aB8GGz8i9cWHtrZT6Pl5k2wYYA78DTDkEKVA%3D%3D; Hm_lpvt_3b316cea958b1e6fb00e41da555d5324=1705473394',
            'Origin': 'https://xy.yzw.cn',
            'Referer': 'https://xy.yzw.cn',
            'Sec-Fetch-Dest': 'empty',
            'Sec-Fetch-Mode': 'cors',
            'Sec-Fetch-Site': 'same-origin',
            'Unique-Id': '9223a6433ca55e55a8a1941d46d7dc28',
            'User-Agent': new_user_agent,
            'X-Auth': 'ZK2Rgq9C7bYXbaK2cR13966U2DYfHj5lna/iQ3EkHwDtLuBfkGGH05NA4ZqwHpT3DpkEpyJtXTWlXf4adKoq4ff53fistW2vYxUhPystHlKJDa6Tm2eopb3IWW1GpcGp4Qe9H3KYaZ+ltEghGnYMTKnoAsJsQHme0Nb2KbNoApg=',
            'X-Yzw-Auth-Token': '7T37OMnWmNbc9GWB9yrP3UC%2FNMdk8NTOHTsBbiBL8RTaALJmTE5mSAGmwG3FySNBxwA6ael7TbSFR%2FTDaE8f4A%3D%3D',
        }
        response = requests.post(self.init_url, headers=headers, json=data, proxies=proxy)
        print('response', response.status_code)
        response.encoding = 'utf-8'
        json_data = response.json()
        print('json_data', json_data)
        return json_data

    def get_total_num(self, data):
        res = data.get('data', None)
        if not res:
            return False, None, None, None
        total_page_num = res.get('totalPage', None)
        total_num = res.get('totalCount', None)
        records = res.get('records', None)
        return True, total_page_num, total_num, records

    def parse_data(self,records):
        for record in records:
            company_name = record.get('name', None)
            company_id = record.get('companyId', None)
            address = record.get('contactAddress', None)
            register_date = record.get('registerDate', None)
            register_fund = record.get('registerFund', None)
            # print('company_name', company_name, company_id, address, register_date, register_fund)
            # redsi去重
            res = self.distinct_id_redis(company_id)
            if res:
                continue
            # 保存mysql
            result = self.save_mysql(company_name, company_id, address, register_date, register_fund)
            if not result:
                continue


    def run(self, num, area_code):
        data = {
            "pageNum": num,
            "pageSize": 100,
            "param": {"registerAreaCode": area_code, }
        }
        try:
            response = yzw.request_get_method(data)
        except Exception as e:
            retry_url_queue_name = 'yzw_retry_queue'
            retry_data = area_code+ '_' + str(num)
            retry_result = self.distinct_retry_data_redis(retry_data)
            if retry_result:
                return False,None # 重复的数据不再重试
            q = connect_message_queue(retry_url_queue_name, url=self.redis_url, maxsize=10000, lazy_limit=True)
            q.put(data)
            return False,None # 重试的数据放入redis

        res, total_page_num, total_num, records = yzw.get_total_num(response)
        if not res:
            return False,None# 没有数据
        yzw.parse_data(records)
        return True, total_page_num

    def __del__(self):
        self.mysql_cursor.close()
        self.mysql_conn.close()



yzw = YzwSpiderProduce()
# '110000000000','370000000000', '410000000000', '120000000000','620000000000','630000000000', '640000000000', '650000000000','7785', '7806','7759','4341','420000000000','430000000000',
# '440000000000', '450000000000','460000000000','500000000000','510000000000', '520000000000','530000000000','540000000000','610000000000', '130000000000','140000000000','150000000000',
# '210000000000','220000000000','230000000000','310000000000','320000000000','330000000000','340000000000', '350000000000',
area_codes = [ '360000000000']
for area_code in area_codes:
    n = 1
    ok,total_num = yzw.run(n, area_code)
    if not ok:
        continue
    for i in range(n+1, total_num+1):
        o,num = yzw.run(i, area_code)
        ran = random.randint(1, 3)
        time.sleep(ran)
        print('第{}页完成'.format(i))
        if not o:
            continue













